/*


Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"context"
	"fmt"
	"iop.inspur.com/constants"
	"iop.inspur.com/util"
	batchv1 "k8s.io/api/batch/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubectl/pkg/drain"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/go-logr/logr"
	"github.com/sirupsen/logrus"
	ckev1alpha1 "iop.inspur.com/api/v1alpha1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
	nodeFinalizerName = "finalizer.stable.node.cke.io"
)

// CkeNodeReconciler reconciles a CkeNode object
type CkeNodeReconciler struct {
	client.Client
	Log    logr.Logger
	Scheme *runtime.Scheme
}

// +kubebuilder:rbac:groups=cke.inspur.com,resources=ckenodes,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=cke.inspur.com,resources=ckenodes/status,verbs=get;update;patch

// --> 查看变化的 ckenode crd
// --> 获取 ckecluster 中的信息，若不止一个，则报错退出
// --> 获取 ckenode 是否要删除的标签
// --> --> 若 ckenode 不为要删除的节点，没有 Finalizers 标签，添加标签到 Finalizers
// --> --> 若 ckenode 为要删除的节点，删除node，创建 job 进行缩容，删除 ckenode 的 Finalizers 标签
// --> 查看 ckenode crd 中的状态
// ---> ---> 不为 complate
// ---> ---> ---> 没有 job 安装；(如果有则删除对应缩容job) 起个 job 进行安装, 标记 node 为 running
// ---> ---> ---> 有 job 安装
// ---> ---> ---> ---> job 状态为 complate，则标记 node 为 complate，删除 job
// ---> ---> ---> ---> job 状态为 failed，  则标记 node 为 fail
// ---> ---> 为 complate； 跳过
func (r *CkeNodeReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
	logrus.Infof("CkeNode Reconcile, CkeNode=%s,    ***************************   NameSpace=%s ", req.NamespacedName, req.Namespace)
	ctx := context.Background()
	var requeue = false

	logrus.Infof("CkeNode Reconcile, CkeNode=%s, get CkeNode. ", req.NamespacedName)

	//get CkeNode info.
	ckenode := &ckev1alpha1.CkeNode{}
	if err := r.Get(ctx, req.NamespacedName, ckenode); err != nil {
		if errors.IsNotFound(err) {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, CkeNode not found. ", req.NamespacedName)
			return ctrl.Result{}, nil
		}
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to get CkeNode.  error=%s", req.NamespacedName, err.Error())
		return ctrl.Result{}, err
	}

	logrus.Infof("CkeNode Reconcile, CkeNode=%s, success to get CkeNode.  address=%s", req.NamespacedName, ckenode.Spec.Address)

	logrus.Infof("CkeNode Reconcile, CkeNode=%s, get CkeNodeList. ", req.NamespacedName)

	//get CkeNodeList info.
	ckenodelist := &ckev1alpha1.CkeNodeList{}
	if err := r.List(ctx, ckenodelist); err != nil {
		if errors.IsNotFound(err) {
			requeue = true
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, CkeNodeList not found. ", req.NamespacedName)
			return ResultIfRequeue(requeue), nil
		}
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to get CkeNodeList.  error=%s", req.NamespacedName, err.Error())
		return ctrl.Result{}, err
	}
	nodes := make([]string, 0)
	for _, ckenode := range ckenodelist.Items {
		nodes = append(nodes, ckenode.Name)
	}
	logrus.Infof("CkeNode Reconcile, CkeNode=%s, success to get CkeNodeList.  allNode=%s", req.NamespacedName, strings.Join(nodes, ", "))

	logrus.Infof("CkeNode Reconcile, CkeNode=%s, get CkeClusterList. ", req.NamespacedName)

	//get CkeCluster
	ckeclusters := &ckev1alpha1.CkeClusterList{}
	if err := r.List(ctx, ckeclusters); err != nil {
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to get CkeClusterList.  error=%s", req.NamespacedName, err.Error())
		return ctrl.Result{}, err
	}
	clusters := make([]string, 0)
	for _, ckecluster := range ckeclusters.Items {
		clusters = append(clusters, ckecluster.Name)
	}
	logrus.Infof("CkeNode Reconcile, CkeNode=%s, success to get CkeClusterList.  allCkeCluster=%s", req.NamespacedName, strings.Join(clusters, ", "))

	if len(ckeclusters.Items) > 1 || len(ckeclusters.Items) < 1 {
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, the count of CkeCluster is not 1. count=%d", len(ckeclusters.Items))
		return ctrl.Result{}, nil
	}
	var ckecluster *ckev1alpha1.CkeCluster
	ckecluster = &ckeclusters.Items[0]
	if ckecluster == nil {
		return ctrl.Result{}, fmt.Errorf("CkeNode Reconcile, CkeNode=%s, not find ckecluster. ", req.NamespacedName)
	}

	//	add finalizers to CkeNode
	if ckenode.DeletionTimestamp.IsZero() {
		if !ContainsStrings(ckenode.Finalizers, nodeFinalizerName) {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, add finalizers to CkeNode. ", req.NamespacedName)
			ckenode.Finalizers = append(ckenode.Finalizers, nodeFinalizerName)
			if err := r.Update(ctx, ckenode); err != nil {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to add finalizers to CkeNode.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}
		}
	}

	//	scale down CkeNode
	if !ckenode.DeletionTimestamp.IsZero() {
		if ContainsStrings(ckenode.Finalizers, nodeFinalizerName) {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, scale down CkeNode. ", req.NamespacedName)

			//缩容节点
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, get KubeNode. ", req.NamespacedName)
			var kubenode = &corev1.Node{}
			var err = r.Get(ctx, types.NamespacedName{Name: ckenode.Name}, kubenode)

			if err != nil && !errors.IsNotFound(err) {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to get KubeNode while deleting CkeNode.   error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, success to get KubeNode.  KubeNode=%s", req.NamespacedName, kubenode.Name)

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Terminating' status. ", req.NamespacedName)

			ckenode.Status.State = TerminateStatus
			r.Update(ctx, ckenode) // 设置节点状态为删除状态，设置不了拉倒

			if err == nil && kubenode.Name == ckenode.Name { // get node, deleted
				var helper = drain.Helper{
					Client:              kubernetes.NewForConfigOrDie(ctrl.GetConfigOrDie()),
					Force:               true,
					GracePeriodSeconds:  0,
					IgnoreAllDaemonSets: true,
					Timeout:             10,
					Out:                 os.Stdout,
					ErrOut:              os.Stdout,
					OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) {
						if pod == nil {
							return
						}
						logrus.Infof("CkeNode Reconcile, CkeNode=%s, deleted or eviction pod (%s), eviction (%t)\n", pod.Name, usingEviction)
					},
				}
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, eviction pod running on CkeNode, ", req.NamespacedName)
				//	驱逐该节点上的pod
				drain.RunNodeDrain(&helper, kubenode.Name)
				//	删除该kubenode节点
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, delete KubeNode, ", req.NamespacedName)
				if err := r.Delete(ctx, kubenode); err != nil {
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to delete KubeNode.  KubeNode=%s error=%s", ckenode.Name, kubenode, err.Error())
					return ctrl.Result{}, err
				}
			}

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, add scale-down label to SpecRoles. ", req.NamespacedName)
			//	add scale-down label to SpecRoles
			for item, node := range ckenodelist.Items {
				if node.Name == ckenode.Name {
					ckenodelist.Items[item].Spec.Roles = append(ckenodelist.Items[item].Spec.Roles, "scale-down")
					break
				}
			}

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, delete all job of scale-up-node. ", req.NamespacedName)

			// fix: 缩容时删除扩容对应的 job
			r.DeleteAllOf(ctx, &batchv1.Job{}, client.InNamespace(req.Namespace), &client.DeleteAllOfOptions{DeleteOptions: constants.BackGroundDeleted}, client.MatchingLabels(map[string]string{
				"app":      ScaleJobAppLabel,
				"nodeName": ckenode.Name,
			}))

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, create scale-down-node job. ", req.NamespacedName)

			scaleDownNodeJob := newScaleDownNodeJob(ckecluster, ckenodelist)

			if scaleDownNodeJob == nil {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, can not create scale-down-node job. possible CkeNode had deleted.", req.NamespacedName)
				return ctrl.Result{}, nil
			}
			// issue: idcentor arm: delete node only delete node, don't do others
			if err := r.Create(ctx, scaleDownNodeJob); err != nil {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to create scale-down-node job.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}

			if len(ckecluster.Spec.ClusterVips) == 0 {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, get CkeComponentList. ", req.NamespacedName)
				ckecomponents := &ckev1alpha1.CkeComponentList{}
				if err := r.List(ctx, ckecomponents); err != nil {
					if errors.IsNotFound(err) {
						requeue = true
						logrus.Infof("CkeNode Reconcile, CkeNode=%s, CkeComponentList not found. ", req.NamespacedName)
						return ResultIfRequeue(requeue), nil
					}
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to get CkeComponentList.  error=%s", req.NamespacedName, err.Error())
					return ctrl.Result{}, err
				}
				needUpdate := false
				for _, ckecomponent := range ckecomponents.Items {
					if strings.Contains(strings.Join(ckecomponent.Status.Nodes, ","), ckenode.Name) {
						ckecomponent.Status.Nodes = RemoveStrings(ckecomponent.Status.Nodes, ckenode.Name)
						needUpdate = true
					}
					if strings.Contains(strings.Join(ckecomponent.Status.ScalingUpPending, ","), ckenode.Name) {
						ckecomponent.Status.ScalingUpPending = RemoveStrings(ckecomponent.Status.ScalingUpPending, ckenode.Name)
						needUpdate = true
					}
					if strings.Contains(strings.Join(ckecomponent.Status.ScalingUpFailed, ","), ckenode.Name) {
						ckecomponent.Status.ScalingUpFailed = RemoveStrings(ckecomponent.Status.ScalingUpFailed, ckenode.Name)
						needUpdate = true
					}
					if strings.Contains(strings.Join(ckecomponent.Status.ScalingDownPending, ","), ckenode.Name) {
						ckecomponent.Status.ScalingDownPending = RemoveStrings(ckecomponent.Status.ScalingDownPending, ckenode.Name)
						needUpdate = true
					}
					if strings.Contains(strings.Join(ckecomponent.Status.ScalingDownFailed, ","), ckenode.Name) {
						ckecomponent.Status.ScalingDownFailed = RemoveStrings(ckecomponent.Status.ScalingDownFailed, ckenode.Name)
						needUpdate = true
					}
					if strings.Contains(strings.Join(ckecomponent.Spec.Nodes, ","), ckenode.Name) {
						ckecomponent.Spec.Nodes = RemoveStrings(ckecomponent.Spec.Nodes, ckenode.Name)
						needUpdate = true
					}
					if needUpdate {
						logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeComponent.  CkeComponent=%s", req.NamespacedName, ckecomponent.Name)
						//	update CkeComponent when CkeNode was delete.
						if err := r.Update(ctx, &ckecomponent); err != nil {
							logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeComponent.  CkeComponent=%s error=%s", req.NamespacedName, ckecomponent.Name, err.Error())
							return ctrl.Result{}, err
						}
					}
				}
			}

			//	delete node by empty Finalizers.
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with Finalizers. ", req.NamespacedName)
			ckenode.ObjectMeta.Finalizers = RemoveStrings(ckenode.Finalizers, nodeFinalizerName)
			if err := r.Update(context.Background(), ckenode); err != nil {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with deleting finalizers.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}

		}
		return ctrl.Result{}, nil
	}

	//set CkeNode role
	if ckenode.Status.CkeRole == "" {
		// 如果 ckerole 为空，则视为新创建 cr。删除 annotation 中的 kubectl 标记，防止下次 apply status 被覆盖
		delete(ckenode.Annotations, corev1.LastAppliedConfigAnnotation)
		ckenode.Status.CkeRole = getNodeRole(ckenode.Spec.Roles)
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with CkeRole. ", req.NamespacedName)
		if err := r.Update(ctx, ckenode); err != nil {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with CkeRole.  error=%s", req.NamespacedName, err.Error())
			return ctrl.Result{}, err
		}
	}

	//set CkeNode ansibleInit
	if ckenode.Status.State == RunningStatus && ckenode.Status.AnsibleInit == false {
		ckenode.Status.AnsibleInit = true
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with AnsibleInit. ", req.NamespacedName)
		if err := r.Update(ctx, ckenode); err != nil {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with ansibleInit.  error=%s", req.NamespacedName, err.Error())
			return ctrl.Result{}, err
		}
	}

	//set CkeNode flags
	if flags := GetCkeNodeFlags(ckenode); flags != ckenode.Status.CkeFlags {
		ckenode.Status.CkeFlags = flags
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with CkeFlags. ", req.NamespacedName)
		if err := r.Update(ctx, ckenode); err != nil {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with CkeFlags.  error=%s", req.NamespacedName, err.Error())
			return ctrl.Result{}, err
		}
	}

	// set label
	if !hasNodeRoleLabel(ckenode.Status.CkeRole, ckenode.Status.Labels) {
		ckenode.Spec.Labels = setNodeRoleLabel(ckenode.Status.CkeRole, ckenode.Spec.Labels)
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with Labels. ", req.NamespacedName)
		if err := r.Update(ctx, ckenode); err != nil {
			logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with Labels.  error=%s", req.NamespacedName, err.Error())
			return ctrl.Result{}, err
		}
	}

	//	scale up CkeNode
	//	initialize CkeNode and install CkeComponent when scale up CkeNode
	if ckenode.Status.State != RunningStatus && ckenode.Status.State != FailedStatus && len(ckecluster.Spec.ClusterVips) == 0 {
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, scale up CkeNode. ", req.NamespacedName)

		logrus.Infof("CkeNode Reconcile, CkeNode=%s, get CkeComponentList. ", req.NamespacedName)
		ckecomponents := &ckev1alpha1.CkeComponentList{}
		if err := r.List(ctx, ckecomponents); err != nil {
			if errors.IsNotFound(err) {
				requeue = true
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, CkeComponentList not found. ",req.NamespacedName)
				return ResultIfRequeue(requeue), nil
			}
			logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to get CkeComponentList.  error=%s", req.NamespacedName, err.Error())
			return ctrl.Result{}, err
		}

		if !ckenode.Status.AnsibleInit {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, CkeNode is not initial. ", req.NamespacedName)
			var CkeJobs = &batchv1.JobList{}
			var JobSelector = map[string]string{
				"app":      ScaleJobAppLabel,
				"nodeName": ckenode.Name,
			}
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, get all of scale-up-node-job. ", req.NamespacedName)
			if err := r.List(ctx, CkeJobs, client.MatchingLabels(JobSelector)); err != nil {
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to get scale-up-node-job.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}

			// 没有 扩容 job，创建新的扩容 job
			if len(CkeJobs.Items) == 0 {
				for idx, node := range ckenodelist.Items {
					if node.Name == ckenode.Name {
						ckenodelist.Items[idx].Spec.Roles = append(ckenodelist.Items[idx].Spec.Roles, constants.ScaleRole)
						logrus.Infof("CkeNode Reconcile, CkeNode=%s, label 'kube-scale' to CkeNode. ", req.NamespacedName)
						break
					}
				}

				logrus.Infof("CkeNode Reconcile, CkeNode=%s, delete all of scale-down-node-job. ", req.NamespacedName)

				// fix: 扩容时删除缩容对应的 job
				r.DeleteAllOf(ctx, &batchv1.Job{}, client.InNamespace(req.Namespace), &client.DeleteAllOfOptions{DeleteOptions: constants.BackGroundDeleted}, client.MatchingLabels(map[string]string{
					"app":      DownJobAppLabel,
					"nodeName": ckenode.Name,
				}))

				logrus.Infof("CkeNode Reconcile, CkeNode=%s, create scale-up-node-job. ", req.NamespacedName)

				if err := r.Create(ctx, newComponentScaleUpAnsibleJob(ckecomponents.Items, ckenodelist.Items, ckecluster)); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to create scale-up-node-job.  error=%s", req.NamespacedName, err.Error())
					return ctrl.Result{}, err
				}

				logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Installing' status. ", req.NamespacedName)

				ckenode.Status.State = InstallStatus
				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with 'Installing' status,  error=%v", req.NamespacedName, err)
				}

				logrus.Infof("CkeNode Reconcile, CkeNode=%s,  *** requeue event ***. ", req.NamespacedName)

				requeue = true
				return ResultIfRequeue(requeue), nil
			} else {
				if JobCompleted(&CkeJobs.Items[0]) {
					ckenode.Status.AnsibleInit = true
					ckenode.Status.Message = ""
					ckenode.Status.Reason = ""
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with AnsibleInit. ", req.NamespacedName)
					if err := r.Update(ctx, ckenode); err != nil {
						logrus.Errorf("CkeNode Reconcile, CkeNode=%s, fail to update CkeNode with AnsibleInit.  error=%s", req.NamespacedName, err.Error())
						return ctrl.Result{}, err
					}
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, delete completed job. ", req.NamespacedName)
					//	TODO:是否可以不删除job，让job自动清除。
					if err := r.Delete(ctx, &CkeJobs.Items[0], &constants.BackGroundDeleted); err != nil {
						logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to delete completed job. JobName=%s error=%s", req.NamespacedName, CkeJobs.Items[0].Name, err.Error())
						return ctrl.Result{}, err
					}

					logrus.Infof("CkeNode Reconcile, CkeNode=%s,  *** requeue event ***. ", req.NamespacedName)
					requeue = true
					return ResultIfRequeue(requeue), nil
				}
				if JobFailed(&CkeJobs.Items[0]) {
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, delete failed job. ", req.NamespacedName)
					// 扩容失败时删除对应 job, 忽略错误
					r.Delete(ctx, &CkeJobs.Items[0], &constants.BackGroundDeleted)

					ckenode.Status.Message = fmt.Sprintf("failed to scale up CkeNode. CkeNode=%s", ckenode.Name)
					ckenode.Status.Reason = "ansible init job failed"
					ckenode.Status.State = FailedStatus
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Failed' status. ", req.NamespacedName)
					if err := r.Update(ctx, ckenode); err != nil {
						logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with 'Failed' status.  error=%s", req.NamespacedName, err.Error())
						return ctrl.Result{}, err
					}
					return ctrl.Result{}, nil
				}

				// 未完成扩容时，更新状态，防止有 job 时状态未被更新
				if ckenode.Status.State == constants.EmptyString {
					ckenode.Status.State = InstallStatus
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Installing' status. ", req.NamespacedName)
					if err := r.Update(ctx, ckenode); err != nil {
						logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with 'Installing' status.  error=%s", req.NamespacedName, err.Error())
					}
				}

				logrus.Infof("CkeNode Reconcile, CkeNode=%s,  *** requeue event ***. ", req.NamespacedName)
				requeue = true
				return ResultIfRequeue(requeue), nil
			}
		}

		ckecomponentsMap := make(map[string]*ckev1alpha1.CkeComponent)
		for i, _ := range ckecomponents.Items {
			ckecomponentsMap[ckecomponents.Items[i].Name] = &ckecomponents.Items[i]
		}
		suBaseComponents := make([]ckev1alpha1.CkeComponent, 0)
		suUnBaseComponents := make([]ckev1alpha1.CkeComponent, 0)

		logrus.Infof("CkeNode Reconcile, CkeNode=%s, filter baseComponent and unBaseComponent. ", req.NamespacedName)

		//	filter baseComponent and unBaseComponent
		for _, ckecomponent := range ckecomponents.Items {
			if ckecomponent.Status.State == ComponentRunningStatus || ckecomponent.Status.State == ComponentScalingUpStatus || ckecomponent.Status.State == ComponentScalingDownStatus {
				if ckecomponent.Labels[BaseComponentLabel] == "true" {
					suBaseComponents = append(suBaseComponents, ckecomponent)
				} else {
					suUnBaseComponents = append(suUnBaseComponents, ckecomponent)
				}
			}
		}

		logrus.Infof("CkeNode Reconcile, CkeNode=%s, sort baseComponents. ", req.NamespacedName)

		//	sort baseComponents
		missingComponent, ckecomponentsSorting := util.CheckAndSort(suBaseComponents)
		if len(missingComponent) != 0 {
			//	TODO:list all of missing components
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, missing baseComponent. ", req.NamespacedName)

			ckenode.Status.Reason = fmt.Sprintf("missing baseComponent.")
			ckenode.Status.Message = fmt.Sprintf("failed to scale ckenode %s, some ckecomponents' depend are not running: %+v.", ckenode.Name, missingComponent)
			ckenode.Status.State = FailedStatus
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Failed' status. ", req.NamespacedName)
			if err := r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with 'Failed' status.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}
			return ctrl.Result{}, nil
		}

		logrus.Infof("CkeNode Reconcile, CkeNode=%s, filter ScaleUpComponents(baseComponents). ", req.NamespacedName)

		//	filter ScaleUpComponents(baseComponents)
		if len(ckenode.Status.ScaleupComponents) == 0 && ckenode.Status.ActiveScaleupComponent == nil && len(ckenode.Status.CompletedScaleupComponents) == 0 {
			sortNeedDeletes := make([]int, 0)
			for i, ckecomponents := range ckecomponentsSorting {
				needDeletes := make([]int, 0)
				for k, ckecomponent := range ckecomponents {
					if !NeedScaleup(ckenode.Spec.Labels, ckecomponentsMap[ckecomponent]) {
						needDeletes = append(needDeletes, k)
					}
				}
				if len(needDeletes) != 0 {
					for n, needDelete := range needDeletes {
						ckecomponents = append(ckecomponents[:needDelete-n], ckecomponents[needDelete-n+1:]...)
					}
				}
				if len(ckecomponents) == 0 {
					sortNeedDeletes = append(sortNeedDeletes, i)
				}
				ckecomponentsSorting[i] = ckecomponents
			}
			if len(sortNeedDeletes) != 0 {
				for n, sortNeedDelete := range sortNeedDeletes {
					ckecomponentsSorting = append(ckecomponentsSorting[:sortNeedDelete-n], ckecomponentsSorting[sortNeedDelete-n+1:]...)
				}
			}

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with ScaleUpComponents. ", req.NamespacedName)

			ckenode.Status.ScaleupComponents = ckecomponentsSorting
			if err := r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with ScaleUpComponents.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}
		}

		logrus.Infof("CkeNode Reconcile, CkeNode=%s, install Components by updating CkeComponent.ScalingUpPending. ", req.NamespacedName)

		//	install Components by update CkeComponent-ScalingUpPending
		requeue, err := r.UpdateAndCheckScaleUpCkeComponents(ctx, ckenode, ckecomponentsMap, suUnBaseComponents)
		if err != nil {
			logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to UpdateAndCheckScaleUpCkeComponents. /%s error=%s", req.Namespace, ckenode.Name, err.Error())
			return ctrl.Result{}, err
		}
		return ResultIfRequeue(requeue), nil
	}

	//	此时为集群已经部署完成；当修改Labels和Roles会触发CkeComponent的动作
	if ckenode.Status.State == RunningStatus {
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, get KubeNode. ", req.NamespacedName)
		//get kubernetes node
		kubenode := &corev1.Node{}
		if err := r.Get(ctx, types.NamespacedName{Name: ckenode.Name}, kubenode); err != nil { // has not been create or other
			requeue = true
			//	TODO: why
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, while label && taint node %s, error: %s", ckenode.Name, err.Error())
			return ResultIfRequeue(requeue), nil
		}

		//update KubeNode taints and CkeNode taints
		toAddtaints := Difference(ckenode.Spec.Taints, ckenode.Status.Taints)
		toDeletetaints := Difference(ckenode.Status.Taints, ckenode.Spec.Taints)
		if len(toAddtaints) != 0 || len(toDeletetaints) != 0 {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update KubeNode taints. ", req.NamespacedName)
			if err := UpdateNodeTaints(r, ctx, kubenode, toAddtaints, toDeletetaints); err != nil {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update KubeNode with taint.  error=%s", ckenode.Name, err.Error())
				//	TODO:test
				if !errors.IsInvalid(err) {
					requeue = true
					return ResultIfRequeue(requeue), nil
				}
				ckenode.Status.Reason = "taints invalid"
				ckenode.Status.Message = "taints configuration error"
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with message of taints invalid. ", req.NamespacedName)
				if err = r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with message of taints invalid.  error=%s", req.NamespacedName, err.Error())
					return ctrl.Result{}, err
				}
				return ctrl.Result{}, err
			}

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode taints. ", req.NamespacedName)
			ckenode.Status.Taints = ckenode.Spec.Taints
			if err := r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with taints.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}
		}
		//	TODO:why
		if ckenode.Status.Reason == "taints invalid" {
			ckenode.Status.Reason = ""
			ckenode.Status.Message = ""
			if err := r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with message.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}
		}

		//	update KubeNode labels and CkeNode labels (scale up or scale down component)
		toAddlabels := Difference(ckenode.Spec.Labels, ckenode.Status.Labels)
		toDeletelabels := Difference(ckenode.Status.Labels, ckenode.Spec.Labels)
		if len(toAddlabels) != 0 || len(toDeletelabels) != 0 {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update KubeNode labels (scale up or scale down component).  KubeNode=%s", req.NamespacedName, kubenode.Name)
			if err := UpdateNodeLabels(r, ctx, kubenode, toAddlabels, toDeletelabels); err != nil {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with Labels.  error=%s", req.NamespacedName, err.Error())
				if !errors.IsInvalid(err) {
					requeue = true
					return ResultIfRequeue(requeue), nil
				}
				ckenode.Status.Reason = "labels invalid"
				ckenode.Status.Message = "labels configuration error"
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with message of labels invalid. ", req.NamespacedName)
				if err = r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with message of labels invalid.  error=%s", req.NamespacedName, err.Error())
					return ctrl.Result{}, err
				}
				return ctrl.Result{}, err
			}

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode labels (scale up or scale down component). ", req.NamespacedName)
			ckenode.Status.Labels = ckenode.Spec.Labels
			if err := r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with Labels.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}

			ckecomponents := &ckev1alpha1.CkeComponentList{}
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, get CkeComponentList. ", req.NamespacedName)
			if err := r.List(ctx, ckecomponents); err != nil {
				if errors.IsNotFound(err) {
					requeue = true
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, CkeComponentList not found. ", req.NamespacedName)
					return ResultIfRequeue(requeue), nil
				}
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to get CkeComponentList.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}

			for _, ckecomponent := range ckecomponents.Items {
				if len(ckecomponent.Spec.Nodes) != 0 || ckecomponent.Spec.Replicas != 0 || (ckecomponent.Status.State != ComponentRunningStatus && ckecomponent.Status.State != ComponentScalingUpStatus && ckecomponent.Status.State != ComponentScalingDownStatus) {
					continue
				}
				needUpate := false
				if ContainsStrings(ckenode.Status.Labels, ckecomponent.Spec.NodeSelector...) {
					if !ContainsStrings(ckecomponent.Status.Nodes, ckenode.Name) && !ContainsStrings(ckecomponent.Status.ScalingUpFailed, ckenode.Name) {
						ckecomponent.Status.ScalingUpPending = AppendIfNotExist(ckecomponent.Status.ScalingUpPending, ckenode.Name)
						needUpate = true
					}
				} else if ContainsStrings(ckecomponent.Status.Nodes, ckenode.Name) && !ContainsStrings(ckecomponent.Status.ScalingDownFailed, ckenode.Name) {
					ckecomponent.Status.ScalingDownPending = AppendIfNotExist(ckecomponent.Status.ScalingDownPending, ckenode.Name)
					needUpate = true
				}
				if needUpate {
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeComponent with ScalingUpPending or ScalingDownPending.  CkeComponent=%s", req.NamespacedName, ckecomponent.Name)
					if err := r.Update(ctx, &ckecomponent); err != nil {
						logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeComponent with ScalingUpPending or ScalingDownPending.  CkeComponent=%s error=%s", req.NamespacedName, ckecomponent.Name, err.Error())
						return ctrl.Result{}, err
					}
				}
			}
		}
		if ckenode.Status.Reason == "labels invalid" {
			ckenode.Status.Reason = ""
			ckenode.Status.Message = ""
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeComponent with deleting 'labels invalid' message. ", req.NamespacedName)
			if err := r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with message.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}
		}

		//保证 ckenode labels 和 taints 不被删除
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, update KubeNode with labels and taints.  KubeNode=%s", req.NamespacedName, kubenode.Name)
		if err := LabelAndTaintNode(r, ctx, kubenode, ckenode.Spec.Labels, ckenode.Spec.Taints); err != nil {
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update KubeNode with labels and taints.  KubeNode=%s error=%s", req.NamespacedName, kubenode.Name, err.Error())
			if !errors.IsInvalid(err) {
				requeue = true
				return ResultIfRequeue(requeue), nil
			}
		}
	}

	// "slave => master" or "master => slave"
	var rolechanges = func() bool {
		return ckenode.Status.CkeRole != getNodeRole(ckenode.Spec.Roles)
	}()

	// TODO: how to judge node is a new node ?
	// weather is a new node, now use ckenode status judge it
	var newnode = func() bool {
		return ckenode.Status.State != RunningStatus
	}()

	// weather install a node already failed
	var installfailed = func() bool {
		return ckenode.Status.State == FailedStatus
	}()

	// make sure there is only one node role-changing at the same time
	if !installfailed && rolechanges {
		for _, node := range ckenodelist.Items {
			if node.Name != ckenode.Name && node.Status.CkeRole != getNodeRole(node.Spec.Roles) {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, only support one node role-changing at the same time.  otherNode=%s error=%s", ckenode.Name, node.Name)
				ckenode.Status.Message = fmt.Sprintf("failed to scale ckenode %s as kubernetes %s", ckenode.Name, getNodeRole(ckenode.Spec.Roles))
				ckenode.Status.Reason = "Only support one node role-changing at the same time"
				// 恢复 ckenode ckerole
				if ckenode.Status.CkeRole == constants.CkeNodeRoleWorker { // 原先为 worker 节点
					ckenode.Spec.Roles = RemoveStrings(ckenode.Spec.Roles, constants.CkeNodeRoleMaster, constants.CkeNodeRoleEtcd)
				} else if ckenode.Status.CkeRole == constants.CkeNodeRoleMaster { // 原先为 master 节点
					ckenode.Spec.Roles = AppendIfNotExist(ckenode.Spec.Roles, constants.CkeNodeRoleMaster, constants.CkeNodeRoleEtcd)
				}
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, recover CkeNode CkeRole. ", req.NamespacedName)
				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with recovering CkeRole.  error=%s", req.NamespacedName, err.Error())
					return ctrl.Result{}, err
				}
				return ctrl.Result{}, nil
			}
		}
	}

	//	len(ckecluster.Spec.ClusterVips) != 0  support old version.
	if !installfailed && (newnode || rolechanges) && len(ckecluster.Spec.ClusterVips) != 0 {
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, new CkeNode or roles changed. ", req.NamespacedName)

		var CkeJobs = &batchv1.JobList{}
		var JobSelector = map[string]string{
			"app":      ScaleJobAppLabel,
			"nodeName": ckenode.Name,
		}
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, get JobList of scale-up-node. ", req.NamespacedName)
		if err := r.List(ctx, CkeJobs, client.MatchingLabels(JobSelector)); err != nil {
			return ctrl.Result{}, err
		}

		// 没有 扩容 job，创建新的扩容 job
		if len(CkeJobs.Items) == 0 {
			for idx, node := range ckenodelist.Items {
				if node.Name == ckenode.Name {
					ckenodelist.Items[idx].Spec.Roles = append(ckenodelist.Items[idx].Spec.Roles, constants.ScaleRole)
					// master 转 slave
					if rolechanges && getNodeRole(ckenode.Spec.Roles) == constants.CkeNodeRoleWorker {
						ckenodelist.Items[idx].Spec.Roles = append(ckenodelist.Items[idx].Spec.Roles, constants.MasterToSlaveRole)
					}
					logrus.Infof("CkeNode Reconcile, CkeNode=%s, roles changed. CkeCluster=%s CkeNodeList=%+v", ckenodelist.Items[idx].Name, ckecluster.Name, ckenodelist.Items)
					break
				}
			}

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, delete all of scale-down-node job. ", req.NamespacedName)
			// fix: 扩容时删除缩容对应的 job
			r.DeleteAllOf(ctx, &batchv1.Job{}, client.InNamespace(req.Namespace), &client.DeleteAllOfOptions{DeleteOptions: constants.BackGroundDeleted}, client.MatchingLabels(map[string]string{
				"app":      DownJobAppLabel,
				"nodeName": ckenode.Name,
			}))

			logrus.Infof("CkeNode Reconcile, CkeNode=%s, create scale-down-node job. ", req.NamespacedName)
			if err := r.Create(ctx, newScaleUpNodeJob(ckecluster, ckenodelist)); err != nil {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to create scale-up-node job.  error=%s", req.NamespacedName, err.Error())
				return ctrl.Result{}, err
			}

			requeue = true
			ckenode.Status.State = InstallStatus
			ckenode.Status.Message = "start a new job for scale node"
			logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Installing' status. ", req.NamespacedName)
			if err := r.Update(ctx, ckenode); err != nil {
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with 'Installing' status.  error=%s", req.NamespacedName, err.Error())
			}
		} else {
			if JobCompleted(&CkeJobs.Items[0]) {
				ckenode.Status.State = RunningStatus
				if rolechanges { // 切换 role 为希望的 role
					ckenode.Status.CkeRole = getNodeRole(ckenode.Spec.Roles)
				}
				ckenode.Status.Message = "this node has been added into cke cluster"

				logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Running' status. ", req.NamespacedName)

				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("fail to update status for node(%s), get error: %s", ckenode.Name, err.Error())
					return ctrl.Result{}, err
				}

				logrus.Infof("CkeNode Reconcile, CkeNode=%s, delete completed scale-up-node job. ", req.NamespacedName)

				if err := r.Delete(ctx, &CkeJobs.Items[0], &constants.BackGroundDeleted); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s, fail to delete completed job(%s), get error: %s", req.NamespacedName, CkeJobs.Items[0].Name, err.Error())
					return ctrl.Result{}, err
				}
				return ResultIfRequeue(requeue), nil // 扩容成功打 labels 和 taints
			}
			if JobFailed(&CkeJobs.Items[0]) {
				// 扩容失败时删除对应 job, 忽略错误
				r.Delete(ctx, &CkeJobs.Items[0], &constants.BackGroundDeleted)

				// when node role changes and failed, change the role to old
				if rolechanges {
					// TODO: role 转化失败后，是否应该给转化为原有状态？
					ckenode.Status.State = RunningStatus
					ckenode.Status.Message = fmt.Sprintf("failed to change ckenode %s role %s to %s", ckenode.Name, getNodeRole(ckenode.Spec.Roles), ckenode.Status.CkeRole)
					ckenode.Status.Reason = ""
					// 恢复 ckenode ckerole
					if ckenode.Status.CkeRole == constants.CkeNodeRoleWorker { // 原先为 worker 节点
						ckenode.Spec.Roles = RemoveStrings(ckenode.Spec.Roles, constants.CkeNodeRoleMaster, constants.CkeNodeRoleEtcd)
					} else if ckenode.Status.CkeRole == constants.CkeNodeRoleMaster { // 原先为 master 节点
						ckenode.Spec.Roles = AppendIfNotExist(ckenode.Spec.Roles, constants.CkeNodeRoleMaster, constants.CkeNodeRoleEtcd)
					}
				} else {
					ckenode.Status.Message = fmt.Sprintf("failed to scale ckenode %s as kubernetes %s", ckenode.Name, getNodeRole(ckenode.Spec.Roles))
					ckenode.Status.Reason = ""
					ckenode.Status.State = FailedStatus
				}
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Running' or 'Failed' status. ", req.NamespacedName)
				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s, fail to update CkeNode with 'Running' or 'Failed' status.  error=%s", req.NamespacedName, err.Error())
					return ctrl.Result{}, err
				}
				return ctrl.Result{}, nil
			}

			// 未完成扩容时，更新状态，防止有 job 时状态未被更新
			if ckenode.Status.State == constants.EmptyString {
				ckenode.Status.State = InstallStatus
				ckenode.Status.Message = "start a new job for scale node"
				logrus.Infof("CkeNode Reconcile, CkeNode=%s, update CkeNode with 'Installing' status. ", req.NamespacedName)
				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to update CkeNode with 'Installing' status.  error=%s", req.NamespacedName, err.Error())
				}
			}

			requeue = true
		}
	}

	//	role changed
	if ckenode.Status.State == RunningStatus && rolechanges && len(ckecluster.Spec.ClusterVips) == 0 {
		var err error
		logrus.Infof("CkeNode Reconcile, CkeNode=%s, role changed, get CkeComponentList.", req.NamespacedName)
		ckecomponents := &ckev1alpha1.CkeComponentList{}
		if err := r.List(ctx, ckecomponents); err != nil {
			logrus.Errorf("CkeNode Reconcile, CkeNode=%s, failed to get CkeComponentList. error=%s", req.NamespacedName, err.Error())
			return ctrl.Result{}, err
		}

		// master 转 slave
		if rolechanges && getNodeRole(ckenode.Spec.Roles) == constants.CkeNodeRoleWorker {
			failedComponents := make([]string, 0)
			rcCkecomponentsName := make([]string, 0)
			rcCkecomponents := make([]ckev1alpha1.CkeComponent, 0)
			for _, ckecomponent := range ckecomponents.Items {
				if ckecomponent.Labels[BaseComponentLabel] == "true" {
					if ContainsStrings(ckecomponent.Status.NodeSelector, constants.NodeMasterLabel) && ContainsStrings(ckecomponent.Status.Nodes, ckenode.Name) {
						if ckecomponent.Status.State != ComponentRunningStatus && ckecomponent.Status.State != ComponentScalingUpStatus && ckecomponent.Status.State != ComponentScalingDownStatus {
							failedComponents = append(failedComponents, ckecomponent.Name)
						} else {
							rcCkecomponentsName = append(rcCkecomponentsName, ckecomponent.Name)
							rcCkecomponents = append(rcCkecomponents, ckecomponent)
						}
					}
				}
			}
			if len(failedComponents) != 0 {
				ckenode.Status.Reason = fmt.Sprintf("ckenode %s master2slave failed", ckenode.Name)
				ckenode.Status.Message = fmt.Sprintf("failed to master2slave ckenode %s, some ckecomponents can't master2slave now: %s.", ckenode.Name, strings.Join(failedComponents, ","))
				ckenode.Spec.Roles = AppendIfNotExist(ckenode.Spec.Roles, constants.CkeNodeRoleMaster, constants.CkeNodeRoleEtcd)
				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s,  %s/%s Update error! err: %s", req.NamespacedName, req.Namespace, ckenode.Name, err.Error())
					return ctrl.Result{}, err
				}
			}
			if len(ckenode.Status.Master2SlaveComponents) == 0 && ckenode.Status.ActiveMaster2SlaveComponent == "" && len(ckenode.Status.CompletedMaster2SlaveComponents) == 0 {
				ckenode.Status.Master2SlaveComponents = rcCkecomponentsName
				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s,  %s/%s Update error! err: %s", req.NamespacedName, req.Namespace, ckenode.Name, err.Error())
					return ctrl.Result{}, err
				}
			}
			requeue, err = r.UpdateAndCheckMaster2SlaveCkeComponents(ctx, ckenode)
			if err != nil {
				if strings.Contains(err.Error(), "recall immediately") {
					return ctrl.Result{Requeue: true, RequeueAfter: 100 * time.Millisecond}, nil
				}
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s,  %s/%s Master2Slave error! err: %s", req.NamespacedName, req.Namespace, ckenode.Name, err.Error())
				return ctrl.Result{}, err
			}
		}

		// slave 转 master
		if rolechanges && getNodeRole(ckenode.Spec.Roles) == constants.CkeNodeRoleMaster {
			failedComponents := make([]string, 0)
			rcCkecomponentsName := make([]string, 0)
			rcCkecomponents := make([]ckev1alpha1.CkeComponent, 0)
			for _, ckecomponent := range ckecomponents.Items {
				if ckecomponent.Labels[BaseComponentLabel] == "true" {
					if ContainsStrings(ckecomponent.Status.NodeSelector, constants.NodeMasterLabel) && ContainsStrings(ckenode.Spec.Labels, RemoveStrings(ckecomponent.Status.NodeSelector, constants.NodeMasterLabel)...) && !ContainsStrings(ckecomponent.Status.Nodes, ckenode.Name) && ckecomponent.Spec.Replicas == 0 {
						if ckecomponent.Status.State != ComponentRunningStatus && ckecomponent.Status.State != ComponentScalingUpStatus && ckecomponent.Status.State != ComponentScalingDownStatus {
							failedComponents = append(failedComponents, ckecomponent.Name)
						} else {
							rcCkecomponentsName = append(rcCkecomponentsName, ckecomponent.Name)
							rcCkecomponents = append(rcCkecomponents, ckecomponent)
						}
					}
				}
			}
			logrus.Infof("slave2master, rcCkecomponents: %s", strings.Join(rcCkecomponentsName, ","))
			if len(failedComponents) != 0 {
				ckenode.Status.Reason = fmt.Sprintf("ckenode %s slave2master failed", ckenode.Name)
				ckenode.Status.Message = fmt.Sprintf("failed to slave2master ckenode %s, some ckecomponents can't slave2master now: %v.", ckenode.Name, failedComponents)
				ckenode.Spec.Roles = RemoveStrings(ckenode.Spec.Roles, constants.CkeNodeRoleMaster, constants.CkeNodeRoleEtcd)
				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s,  %s/%s Update error! err: %s", req.NamespacedName, req.Namespace, ckenode.Name, err.Error())
					return ctrl.Result{}, err
				}
			}
			if len(ckenode.Status.Slave2MasterComponents) == 0 && ckenode.Status.ActiveSlave2MasterComponent == "" && len(ckenode.Status.CompletedSlave2MasterComponents) == 0 {
				ckenode.Status.Slave2MasterComponents = rcCkecomponentsName
				if err := r.Update(ctx, ckenode); err != nil {
					logrus.Errorf("CkeNode Reconcile, CkeNode=%s,  %s/%s Update error! err: %s", req.NamespacedName, req.Namespace, ckenode.Name, err.Error())
					return ctrl.Result{}, err
				}
			}
			requeue, err = r.UpdateAndCheckSlave2MasterCkeComponents(ctx, ckenode)
			if err != nil {
				if strings.Contains(err.Error(), "recall immediately") {
					return ctrl.Result{Requeue: true, RequeueAfter: 100 * time.Millisecond}, nil
				}
				logrus.Errorf("CkeNode Reconcile, CkeNode=%s,  %s/%s Slave2Master error! err: %s", req.NamespacedName, req.Namespace, ckenode.Name, err.Error())
				return ctrl.Result{}, err
			}
		}
	}

	return ResultIfRequeue(requeue), nil
}

func (r *CkeNodeReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&ckev1alpha1.CkeNode{}).
		Complete(r)
}

func (r *CkeNodeReconciler) UpdateAndCheckScaleUpCkeComponents(ctx context.Context, ckenode *ckev1alpha1.CkeNode, ckecomponentsMap map[string]*ckev1alpha1.CkeComponent, suUnBaseComponents []ckev1alpha1.CkeComponent) (bool, error) {
	requeue := false
	var err error

	if len(ckenode.Status.ScaleupComponents) > 0 && ckenode.Status.ActiveScaleupComponent == nil {
		err := r.AddScaleUpActiveCkeComponents(ctx, ckenode, ckecomponentsMap)
		if err != nil {
			return requeue, err
		}
		requeue = true
		return requeue, err
	}

	failedComponents := make([]string, 0)
	completedComponents := make([]string, 0)
	scaleupingComponents := make([]string, 0)
	for _, component := range ckenode.Status.ActiveScaleupComponent {
		ComponentScaleUpCompleted, ComponentScaleUpFailed := CheckScaleUpCkeComponent(ckenode.Name, ckecomponentsMap[component])
		if ComponentScaleUpFailed {
			failedComponents = append(failedComponents, ckecomponentsMap[component].Name)
		}
		if ComponentScaleUpCompleted {
			completedComponents = append(completedComponents, ckecomponentsMap[component].Name)
		}
		scaleupingComponents = append(scaleupingComponents, ckecomponentsMap[component].Name)
	}

	if len(failedComponents) != 0 {
		if len(failedComponents)+len(completedComponents) == len(ckenode.Status.ActiveScaleupComponent) {
			ckenode.Status.Reason = fmt.Sprintf("ckenode %s create failed", ckenode.Name)
			ckenode.Status.Message = fmt.Sprintf("ckecomponents %s scale up failed ", strings.Join(failedComponents, ","))
			ckenode.Status.ScaleupComponents = append([][]string{ckenode.Status.ActiveScaleupComponent}, ckenode.Status.ScaleupComponents...)
			ckenode.Status.ActiveScaleupComponent = nil
			ckenode.Status.State = FailedStatus
			if err := r.Update(ctx, ckenode); err != nil {
				return requeue, err
			}
			return requeue, err
		} else {
			ckenode.Status.Reason = fmt.Sprintf("ckenode %s create failed, and some ckecomponents are still scale up", ckenode.Name)
			ckenode.Status.Message = fmt.Sprintf("ckecomponents %v are still scale up; ckecomponents %v scale up failed ", strings.Join(scaleupingComponents, ","), strings.Join(failedComponents, ","))
			if err := r.Update(ctx, ckenode); err != nil {
				return requeue, err
			}
			requeue = true
			return requeue, err
		}
	}

	if len(completedComponents) == len(ckenode.Status.ActiveScaleupComponent) {
		ckenode.Status.CompletedScaleupComponents = append(ckenode.Status.CompletedScaleupComponents, ckenode.Status.ActiveScaleupComponent)
		ckenode.Status.ActiveScaleupComponent = nil
		if err = r.Update(ctx, ckenode); err != nil {
			logrus.Errorf("UpdateAndCheckScaleUpCkeComponents, %s/%s Update error! err: %s", ckenode.Namespace, ckenode.Name, err.Error())
			return requeue, err
		}
		if len(ckenode.Status.ScaleupComponents) == 0 {
			ckenode.Status.CompletedScaleupComponents = nil
			ckenode.Status.State = RunningStatus
			ckenode.Status.Message = "this node has been added into cke cluster"
			if err = r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("UpdateAndCheckScaleUpCkeComponents, %s/%s Update error! err: %s", ckenode.Namespace, ckenode.Name, err.Error())
				return requeue, err
			}
			for _, suUnBaseComponent := range suUnBaseComponents {
				if NeedScaleup(ckenode.Spec.Labels, &suUnBaseComponent) {
					suUnBaseComponent.Status.ScalingUpPending = AppendIfNotExist(suUnBaseComponent.Status.ScalingUpPending, ckenode.Name)
					if err = r.Update(ctx, &suUnBaseComponent); err != nil {
						logrus.Errorf("UpdateAndCheckScaleUpCkeComponents, ckecomponent %s/%s Update error! err: %s", suUnBaseComponent.Namespace, suUnBaseComponent.Name, err.Error())
						return requeue, err
					}
				}
			}
		}
	}

	requeue = true
	return requeue, err
}

func (r *CkeNodeReconciler) AddScaleUpActiveCkeComponents(ctx context.Context, ckenode *ckev1alpha1.CkeNode, ckecomponentsMap map[string]*ckev1alpha1.CkeComponent) error {
	var err error

	ckenode.Status.ActiveScaleupComponent = ckenode.Status.ScaleupComponents[0]
	if len(ckenode.Status.ScaleupComponents) > 1 {
		ckenode.Status.ScaleupComponents = ckenode.Status.ScaleupComponents[1:]
	} else {
		ckenode.Status.ScaleupComponents = nil
	}

	for _, component := range ckenode.Status.ActiveScaleupComponent {
		ckecomponentsMap[component].Status.ScalingUpPending = append(ckecomponentsMap[component].Status.ScalingUpPending, ckenode.Name)
		if err = r.Update(ctx, ckecomponentsMap[component]); err != nil {
			logrus.Errorf("CkeNode Reconcile, failed to update CkeComponent with AddActiveCkeComponents.  CkeComponent=%s/%s error=%s", ckenode.Name, ckecomponentsMap[component].Namespace, ckecomponentsMap[component].Name, err.Error())
			return err
		}
	}

	if err = r.Update(ctx, ckenode); err != nil {
		logrus.Errorf("CkeNode Reconcile, failed to update CkeNode with ScaleUpComponents and ActiveScaleUpComponent. CkeNode=%s/%s error=%s", ckenode.Namespace, ckenode.Name, err.Error())
		return err
	}

	return err
}

func (r *CkeNodeReconciler) UpdateAndCheckMaster2SlaveCkeComponents(ctx context.Context, ckenode *ckev1alpha1.CkeNode) (bool, error) {
	requeue := false
	var err error

	if len(ckenode.Status.Master2SlaveComponents) > 0 && ckenode.Status.ActiveMaster2SlaveComponent == "" {
		err := r.AddMaster2SlaveActiveCkeComponents(ctx, ckenode)
		if err != nil {
			return requeue, err
		}
		requeue = true
		return requeue, err
	}

	namespacedName := types.NamespacedName{
		Namespace: ckenode.Namespace,
		Name:      ckenode.Status.ActiveMaster2SlaveComponent,
	}
	activeComponent := &ckev1alpha1.CkeComponent{}
	if err = r.Get(ctx, namespacedName, activeComponent); err != nil {
		logrus.Errorf("UpdateAndCheckMaster2SlaveCkeComponents, get CkeComponent %s/%s error! err: %s", ckenode.Namespace, ckenode.Status.ActiveMaster2SlaveComponent, err.Error())
		requeue = true
		return requeue, err
	}

	ComponentMaster2SlaveCompleted, ComponentMaster2SlaveFailed, err := CheckMaster2SlaveCkeComponent(ckenode.Name, activeComponent)
	if err != nil {
		return requeue, err
	}

	if ComponentMaster2SlaveFailed {
		ckenode.Status.Reason = fmt.Sprintf("ckenode %s master2slave failed", ckenode.Name)
		ckenode.Status.Message = fmt.Sprintf("ckecomponent %s scaledown failed, See log in /var/log/%s/%s/scaledown for details.", activeComponent.Name, activeComponent.Spec.ClusterName, activeComponent.Name)
		ckenode.Status.Master2SlaveComponents = append([]string{ckenode.Status.ActiveMaster2SlaveComponent}, ckenode.Status.Master2SlaveComponents...)
		ckenode.Status.ActiveMaster2SlaveComponent = ""
		ckenode.Spec.Roles = AppendIfNotExist(ckenode.Spec.Roles, constants.CkeNodeRoleMaster, constants.CkeNodeRoleEtcd)
		if err := r.Update(ctx, ckenode); err != nil {
			return requeue, err
		}
	}

	if ComponentMaster2SlaveCompleted {
		ckenode.Status.CompletedMaster2SlaveComponents = append(ckenode.Status.CompletedMaster2SlaveComponents, ckenode.Status.ActiveMaster2SlaveComponent)
		ckenode.Status.ActiveMaster2SlaveComponent = ""
		if err = r.Update(ctx, ckenode); err != nil {
			logrus.Errorf("UpdateAndCheckMaster2SlaveCkeComponents, %s/%s Update error! err: %s", ckenode.Namespace, ckenode.Name, err.Error())
			return requeue, err
		}
		if len(ckenode.Status.Master2SlaveComponents) == 0 {
			ckenode.Status.CompletedMaster2SlaveComponents = nil
			ckenode.Status.CkeRole = getNodeRole(ckenode.Spec.Roles)
			ckenode.Status.Message = "this node has been added into cke cluster"
			if err = r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("UpdateAndCheckMaster2SlaveCkeComponents, %s/%s Update error! err: %s", ckenode.Namespace, ckenode.Name, err.Error())
				return requeue, err
			}
			return requeue, fmt.Errorf("recall immediately")
		}
	}

	requeue = true
	return requeue, err
}

func (r *CkeNodeReconciler) AddMaster2SlaveActiveCkeComponents(ctx context.Context, ckenode *ckev1alpha1.CkeNode) error {
	var err error

	ckenode.Status.ActiveMaster2SlaveComponent = ckenode.Status.Master2SlaveComponents[0]
	if len(ckenode.Status.Master2SlaveComponents) > 1 {
		ckenode.Status.Master2SlaveComponents = ckenode.Status.Master2SlaveComponents[1:]
	} else {
		ckenode.Status.Master2SlaveComponents = nil
	}

	namespacedName := types.NamespacedName{
		Namespace: ckenode.Namespace,
		Name:      ckenode.Status.ActiveMaster2SlaveComponent,
	}
	activeComponent := &ckev1alpha1.CkeComponent{}
	if err = r.Get(ctx, namespacedName, activeComponent); err != nil {
		return err
	}

	if !ContainsStrings(activeComponent.Status.ScalingDownFailed, ckenode.Name) {
		activeComponent.Status.ScalingDownPending = AppendIfNotExist(activeComponent.Status.ScalingDownPending, ckenode.Name)
		if err = r.Update(ctx, activeComponent); err != nil {
			logrus.Errorf("AddMaster2SlaveActiveCkeComponents, ckecomponent %s/%s Update error! err: %s", activeComponent.Namespace, activeComponent.Name, err.Error())
			return err
		}
	}

	if err = r.Update(ctx, ckenode); err != nil {
		logrus.Errorf("AddMaster2SlaveActiveCkeComponents, ckenode %s/%s Update error! err: %s", ckenode.Namespace, ckenode.Name, err.Error())
		return err
	}
	return err
}

func (r *CkeNodeReconciler) UpdateAndCheckSlave2MasterCkeComponents(ctx context.Context, ckenode *ckev1alpha1.CkeNode) (bool, error) {
	requeue := false
	var err error
	if len(ckenode.Status.Slave2MasterComponents) > 0 && ckenode.Status.ActiveSlave2MasterComponent == "" {
		err := r.AddSlave2MasterActiveCkeComponents(ctx, ckenode)
		if err != nil {
			return requeue, err
		}
		requeue = true
		return requeue, err
	}

	namespacedName := types.NamespacedName{
		Namespace: ckenode.Namespace,
		Name:      ckenode.Status.ActiveSlave2MasterComponent,
	}
	activeComponent := &ckev1alpha1.CkeComponent{}
	if err = r.Get(ctx, namespacedName, activeComponent); err != nil {
		logrus.Errorf("UpdateAndCheckSlave2MasterCkeComponents, get CkeComponent %s/%s error! err: %s", ckenode.Namespace, ckenode.Status.ActiveSlave2MasterComponent, err.Error())
		requeue = true
		return requeue, err
	}

	ComponentSlave2MasterCompleted, ComponentSlave2MasterFailed, err := CheckSlave2MasterCkeComponent(ckenode.Name, activeComponent)
	if err != nil {
		return requeue, err
	}

	if ComponentSlave2MasterFailed {
		ckenode.Status.Reason = fmt.Sprintf("ckenode %s slave2master failed", ckenode.Name)
		ckenode.Status.Message = fmt.Sprintf("ckecomponents %v slave2master failed, See log in /var/log/%s/%s/scaleup for details.", activeComponent.Name, activeComponent.Spec.ClusterName, activeComponent.Name)
		ckenode.Status.Slave2MasterComponents = append([]string{ckenode.Status.ActiveSlave2MasterComponent}, ckenode.Status.Slave2MasterComponents...)
		ckenode.Status.ActiveSlave2MasterComponent = ""
		ckenode.Spec.Roles = RemoveStrings(ckenode.Spec.Roles, constants.CkeNodeRoleMaster, constants.CkeNodeRoleEtcd)
		if err := r.Update(ctx, ckenode); err != nil {
			return requeue, err
		}
		return requeue, err
	}

	if ComponentSlave2MasterCompleted {
		ckenode.Status.CompletedSlave2MasterComponents = append(ckenode.Status.CompletedSlave2MasterComponents, ckenode.Status.ActiveSlave2MasterComponent)
		ckenode.Status.ActiveSlave2MasterComponent = ""
		if err = r.Update(ctx, ckenode); err != nil {
			logrus.Errorf("UpdateAndCheckSlave2MasterCkeComponents, %s/%s Update error! err: %s", ckenode.Namespace, ckenode.Name, err.Error())
			return requeue, err
		}
		if len(ckenode.Status.Slave2MasterComponents) == 0 {
			ckenode.Status.CompletedSlave2MasterComponents = nil
			ckenode.Status.CkeRole = getNodeRole(ckenode.Spec.Roles)
			ckenode.Status.Message = "this node has been added into cke cluster"
			if err = r.Update(ctx, ckenode); err != nil {
				logrus.Errorf("UpdateAndCheckSlave2MasterCkeComponents, %s/%s Update error! err: %s", ckenode.Namespace, ckenode.Name, err.Error())
				return requeue, err
			}
			return requeue, fmt.Errorf("recall immediately")
		}
	}

	requeue = true
	return requeue, err
}

func (r *CkeNodeReconciler) AddSlave2MasterActiveCkeComponents(ctx context.Context, ckenode *ckev1alpha1.CkeNode) error {
	var err error

	ckenode.Status.ActiveSlave2MasterComponent = ckenode.Status.Slave2MasterComponents[0]
	if len(ckenode.Status.Slave2MasterComponents) > 1 {
		ckenode.Status.Slave2MasterComponents = ckenode.Status.Slave2MasterComponents[1:]
	} else {
		ckenode.Status.Slave2MasterComponents = nil
	}

	namespacedName := types.NamespacedName{
		Namespace: ckenode.Namespace,
		Name:      ckenode.Status.ActiveSlave2MasterComponent,
	}
	activeComponent := &ckev1alpha1.CkeComponent{}
	if err = r.Get(ctx, namespacedName, activeComponent); err != nil {
		return err
	}

	if !ContainsStrings(activeComponent.Status.ScalingUpFailed, ckenode.Name) {
		activeComponent.Status.ScalingUpPending = AppendIfNotExist(activeComponent.Status.ScalingUpPending, ckenode.Name)
		if err = r.Update(ctx, activeComponent); err != nil {
			logrus.Errorf("AddSlave2MasterActiveCkeComponents, ckecomponent %s/%s Update error! err: %s", activeComponent.Namespace, activeComponent.Name, err.Error())
			return err
		}
	}

	if err = r.Update(ctx, ckenode); err != nil {
		logrus.Errorf("AddSlave2MasterActiveCkeComponents, ckenode %s/%s Update error! err: %s", ckenode.Namespace, ckenode.Name, err.Error())
		return err
	}

	return err
}

func CheckScaleUpCkeComponent(ckenode string, ckecomponent *ckev1alpha1.CkeComponent) (bool, bool) {
	ComponentScaleUpCompleted := false
	ComponentScaleUpFailed := false
	if (ckecomponent.Status.State == ComponentRunningStatus || ckecomponent.Status.State == ComponentScalingUpStatus || ckecomponent.Status.State == ComponentScalingDownStatus) && ContainsStrings(ckecomponent.Status.Nodes, ckenode) {
		ComponentScaleUpCompleted = true
	}
	if strings.Contains(strings.Join(ckecomponent.Status.ScalingUpFailed, ","), ckenode) {
		ComponentScaleUpFailed = true
	}
	return ComponentScaleUpCompleted, ComponentScaleUpFailed
}

func CheckMaster2SlaveCkeComponent(ckenode string, ckecomponent *ckev1alpha1.CkeComponent) (bool, bool, error) {
	ComponentMaster2SlaveCompleted := false
	ComponentMaster2SlaveFailed := false
	if (ckecomponent.Status.State == ComponentRunningStatus || ckecomponent.Status.State == ComponentScalingUpStatus || ckecomponent.Status.State == ComponentScalingDownStatus) && !ContainsStrings(ckecomponent.Status.Nodes, ckenode) {
		ComponentMaster2SlaveCompleted = true
	}
	if strings.Contains(strings.Join(ckecomponent.Status.ScalingDownFailed, ","), ckenode) {
		ComponentMaster2SlaveFailed = true
	}
	return ComponentMaster2SlaveCompleted, ComponentMaster2SlaveFailed, nil
}

func CheckSlave2MasterCkeComponent(ckenode string, ckecomponent *ckev1alpha1.CkeComponent) (bool, bool, error) {
	ComponentSlave2MasterCompleted := false
	ComponentSlave2MasterFailed := false
	if (ckecomponent.Status.State == ComponentRunningStatus || ckecomponent.Status.State == ComponentScalingUpStatus || ckecomponent.Status.State == ComponentScalingDownStatus) && ContainsStrings(ckecomponent.Status.Nodes, ckenode) {
		ComponentSlave2MasterCompleted = true
	}
	if strings.Contains(strings.Join(ckecomponent.Status.ScalingUpFailed, ","), ckenode) {
		ComponentSlave2MasterFailed = true
	}
	return ComponentSlave2MasterCompleted, ComponentSlave2MasterFailed, nil
}

func GetCkeNodeFlags(node *ckev1alpha1.CkeNode) (flags string) {
	var flagSlice = make([]string, 0)

	for annotation, value := range node.Annotations {
		if regexp.MustCompile(fmt.Sprintf("%s/%s", constants.CkeFlagAnnotation, ".*")).MatchString(annotation) {
			if exist, err := strconv.ParseBool(value); err == nil && exist {
				flagSlice = AppendIfNotExist(flagSlice, strings.Split(annotation, "/")[1])
			}
		}
	}

	if len(flagSlice) == 0 {
		return constants.CkeFlagNone
	}

	sort.Strings(flagSlice) // 排序，防止两次结果不一样
	return SpliceStringArrary(flagSlice, ",", false, false)
}

func SpliceStringArrary(arrary []string, connector string, withhead bool, withend bool) (result string) {
	if len(arrary) == 0 {
		return
	}

	var head, end string // head & end
	if withhead {
		head = connector
	}
	if withend {
		end = connector
	}

	for item := range arrary {
		result = fmt.Sprintf("%s%s%s", result, connector, arrary[item])
	}

	return fmt.Sprintf("%s%s%s", head, result[len(connector):], end)
}

func UpdateNodeTaints(r client.Client, ctx context.Context, kubenode *corev1.Node, toAddtaints []string, toDeletetaints []string) error {
	for _, toDeletetaint := range toDeletetaints {
		if kubetaint, suss := GetTaintKeyValue(toDeletetaint); suss && TaintExist(kubenode.Spec.Taints, kubetaint) {
			for i, v := range kubenode.Spec.Taints {
				if v.Key == kubetaint.Key && v.Value == kubetaint.Value && v.Effect == kubetaint.Effect {
					kubenode.Spec.Taints = append(kubenode.Spec.Taints[:i], kubenode.Spec.Taints[i+1:]...)
				}
			}
		}
	}

	for _, toAddtaint := range toAddtaints {
		if kubetaint, suss := GetTaintKeyValue(toAddtaint); suss && !TaintExist(kubenode.Spec.Taints, kubetaint) {
			if len(kubenode.Spec.Taints) == 0 {
				taints := make([]corev1.Taint, 0)
				taints = append(taints, kubetaint)
				kubenode.Spec.Taints = taints
				continue
			}
			kubenode.Spec.Taints = append(kubenode.Spec.Taints, kubetaint)
		}
	}

	return r.Update(ctx, kubenode)
}

func UpdateNodeLabels(r client.Client, ctx context.Context, kubenode *corev1.Node, toAddlabels []string, toDeletelabels []string) error {
	for _, toDeletelabel := range toDeletelabels {
		if key, _, suss := GetLabelKeyValue(toDeletelabel); suss {
			delete(kubenode.Labels, key)
		}
	}
	for _, toAddlabel := range toAddlabels {
		if key, value, suss := GetLabelKeyValue(toAddlabel); suss {
			if len(kubenode.Labels) == 0 {
				labels := make(map[string]string)
				labels[key] = value
				kubenode.Labels = labels
				continue
			}
			kubenode.Labels[key] = value
		}
	}

	return r.Update(ctx, kubenode)
}

func LabelAndTaintNode(r client.Client, ctx context.Context, kubenode *corev1.Node, labels []string, taints []string) error {
	for _, label := range labels {
		if key, value, suss := GetLabelKeyValue(label); suss {
			kubenode.Labels[key] = value
		}
	}

	for _, taint := range taints {
		if kubetaint, suss := GetTaintKeyValue(taint); suss && !TaintExist(kubenode.Spec.Taints, kubetaint) {
			kubenode.Spec.Taints = append(kubenode.Spec.Taints, kubetaint)
		}
	}

	return r.Update(ctx, kubenode)
}

func GetLabelKeyValue(label string) (key string, value string, suss bool) {
	if strings.Count(label, "=") != 1 {
		return
	}
	return strings.Split(label, "=")[0], strings.Split(label, "=")[1], true
}

func GetTaintKeyValue(taint string) (kubetaint corev1.Taint, suss bool) {
	if !regexp.MustCompile(".*=.*:.*").MatchString(taint) || strings.Count(taint, "=") != 1 || strings.Count(taint, ":") != 1 {
		return
	}

	var key = strings.Split(taint, "=")[0]
	var value = strings.Split(strings.Split(taint, "=")[1], ":")[0]
	var effect = corev1.TaintEffect(strings.Split(taint, ":")[1])

	if ValidateTaintEffect(effect) != nil {
		return
	}

	kubetaint.Key = key
	kubetaint.Value = value
	kubetaint.Effect = effect

	return kubetaint, true
}

func TaintExist(taints []corev1.Taint, taint corev1.Taint) bool {
	for _, intaint := range taints {
		if fmt.Sprintf("%s=%s:%s", intaint.Key, intaint.Value, intaint.Effect) == fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect) {
			return true
		}
	}
	return false
}

func ValidateTaintEffect(effect corev1.TaintEffect) error {
	if effect != corev1.TaintEffectNoSchedule && effect != corev1.TaintEffectPreferNoSchedule && effect != corev1.TaintEffectNoExecute {
		return fmt.Errorf("invalid taint effect: %v, unsupported taint effect", effect)
	}

	return nil
}

func ResultIfRequeue(requeue bool) (result ctrl.Result) {
	if requeue {
		result.Requeue = requeue
		result.RequeueAfter = RequeueAfter * time.Second
	}

	return
}

func getNodeRole(roles []string) (role string) {
	// default roles is kubernetes roles
	role = constants.CkeNodeRoleWorker

	if ContainsStrings(roles, constants.CkeNodeRoleMaster) {
		role = constants.CkeNodeRoleMaster
	}
	return role
}

func hasNodeRoleLabel(role string, labels []string) bool {
	switch role {
	case constants.CkeNodeRoleMaster:
		return ContainsStrings(labels, constants.NodeMasterLabel) && ContainsStrings(labels, constants.NodeWorkerLabel)
	case constants.CkeNodeRoleWorker:
		return !ContainsStrings(labels, constants.NodeMasterLabel) && ContainsStrings(labels, constants.NodeWorkerLabel)
	default:
		return true
	}
}

func setNodeRoleLabel(role string, labels []string) []string {
	switch role {
	case constants.CkeNodeRoleMaster:
		return AppendIfNotExist(labels, constants.NodeMasterLabel, constants.NodeWorkerLabel)
	case constants.CkeNodeRoleWorker:
		return AppendIfNotExist(RemoveStrings(labels, constants.NodeMasterLabel), constants.NodeWorkerLabel)
	default:
		return labels
	}
}

func NeedScaleup(labels []string, ckecomponent *ckev1alpha1.CkeComponent) bool {
	return len(ckecomponent.Spec.Nodes) == 0 && ckecomponent.Spec.Replicas == 0 && ContainsStrings(labels, ckecomponent.Spec.NodeSelector...)
}
